#!/usr/bin/python3
# -*- coding:utf-8 -*-
'''
@author Linson
@email studyneverlate@126.com
@qq 691000737
this python module is working fine for python3.x
used to grab web page by using css selector methods.
Based on basic python module as re,os,time,random,urllib.request.
usage:
    pls reffering to simple_html_dom.php,their web sites list all methods that class
    support. same as this python module do!
'''
HDOM_TYPE_ELEMENT=1
HDOM_TYPE_COMMENT=2
HDOM_TYPE_TEXT=   3
HDOM_TYPE_ENDTAG= 4
HDOM_TYPE_ROOT=   5
HDOM_TYPE_UNKNOWN=6
HDOM_QUOTE_DOUBLE=0
HDOM_QUOTE_SINGLE=1
HDOM_QUOTE_NO=    3
HDOM_INFO_BEGIN=  0
HDOM_INFO_END=    1
HDOM_INFO_QUOTE=  2
HDOM_INFO_SPACE=  3
HDOM_INFO_TEXT=   4
HDOM_INFO_INNER=  5
HDOM_INFO_OUTER=  6
HDOM_INFO_ENDSPACE=7
import re
import os
import requests
# helper defs
# -----------------------------------------------------------------------------
# define a function to get file content only(url is allowed)
def loadData(filename,encode='utf-8',m='GET',dataParam={}):
    if filename.startswith('http://') or filename.startswith('https://') or filename.startswith('ftp://'):
        try:
            if m=='POST':
                headers = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain","User-Agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/30.0.1599.114 Chrome/30.0.1599.114 Safari/537.36"}
                f=requests.post(filename,timeout=60,headers=headers,data=dataParam)
                data=f.content.decode(encode)
            else:
                headers = {"User-Agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/30.0.1599.114 Chrome/30.0.1599.114 Safari/537.36"}
                f=requests.get(filename,timeout=60,headers=headers,params=dataParam)
                data=f.content.decode(encode)
        except:
            print('get page failed')
            data=''
    if os.path.isfile(filename):
        try:
            with open(filename) as f:
                data=f.read().decode(encode)
                f.close()
        except:
            data=''

    return data

# get html dom form file
def file_get_html(filename,encoding='utf-8',m='GET',dataParam={}):
    dom = simple_html_dom()
    data=loadData(filename,encoding,m,dataParam)
    dom.load(data, True)
    return dom


# get html dom form string
def str_get_html(_str, lowercase=True):
    dom = simple_html_dom()
    dom.load(_str, lowercase)
    return dom


# dump html dom tree
def dump_html_tree(node, show_attr=True, deep=0):
    lead = '    '*deep
    print(lead+node.tag)
    if show_attr and len(node.attr)>0:
        print('(')
        for k in node.attr.keys():
            print("["+k+"]=>\""+node.attr.get(k)+'", ')
        print(')')

    print("\n")
    for c in node:
        dump_html_tree(c, show_attr, deep+1)
# deprecated
def file_get_dom(filename):
    return file_get_html(filename)

# deprecated
def str_get_dom(_str,lowercase=True):
    return str_get_html(_str,lowercase)

def strspn(str1,str2,start=0):
    count=0
    if str2==None or str1==None:
        return count
    for i in str1[start:]:
        if i in str2:
            count+=1
        else:
            return count
def strcspn(str1,str2,start=0):
    #pass
    count=0
    for i in str1[start:]:
        if i not in str2:
            count+=1
        else:
            return count


# simple html dom node
# -----------------------------------------------------------------------------
class simple_html_dom_node(object):
    ''' Simple Html Dom Node ,used for SimpleHtmlDom Class'''

    def __init__(self,dom):
        self.nodetype = HDOM_TYPE_TEXT
        self.tag = 'text'
        self.attr = {}
        self.children = []
        self.nodes = []
        self.parent = None
        self._ = {}
        self.dom = dom
        #print(dom.lowercase)
        self.__id__=id(self)
        self.dom.nodes.append(self)
    def __del__(self):
        self.clear()
        #TEST
        #print('Object Delelted')
        #object.__del__(self)


    def __str__(self):
        return self.outertext()


    # clean up memory due to php5 circular references memory leak...
    def clear(self):
        #print('Object Delelted')
        self.dom = None
        self.nodes = []
        self.parent = None
        self.children = []


    # dump node's tree
    def dump(self,show_attr=True):
        dump_html_tree(self, show_attr)


    # returns the parent of node
    def get_parent(self):
        return self.parent


    # returns children of node
    def child(self,idx=-1):
        if idx==-1:
            return self.children
        try:
            if self.children[idx]!=None:
                return self.children[idx]
        except:
            return None


    # returns the first child of node
    def first_child(self):
        if len(self.children)>0:
            return self.children[0]
        return None


    # returns the last child of node
    def last_child(self):
        count=len(self.children)
        if count>0:
            return self.children[-1]
        return None


    # returns the next sibling of node
    def next_sibling(self):
        if self.parent==None:
            return None
        idx = 0
        count = len(self.parent.children)
        ## python object ,need define __eq__ ,__ne__
        while idx<count and self!=self.parent.children[idx]:
            idx+=1
        idx+=1
        if idx>=count:
            return None
        return self.parent.children[idx]


    # returns the previous sibling of node
    def prev_sibling(self):
        if self.parent==None:
            return None
        idx = 0
        count = len(self.parent.children)
        while idx<count and self!=self.parent.children[idx]:
            idx+=1
        idx-=1
        if idx<0:
            return None
        return self.parent.children[idx]


    # get dom node's inner html
    def innertext(self):
        if self._.get(HDOM_INFO_INNER):
            return self._[HDOM_INFO_INNER]
        if self._.get(HDOM_INFO_TEXT):
            return self.dom.restore_noise(self._[HDOM_INFO_TEXT])

        ret=''
        for n in self.nodes:
            ret+=n.outertext()
        return ret


    # get dom node's outer text (with tag)
    def outertext(self):
        if self.tag=='root':
            return self.innertext()

        # trigger callback
        if self.dom.callback!=None:
            #?? Attribute Type html_dom_node
            self.dom.callback(self)

        if self._.get(HDOM_INFO_OUTER):
            return self._[HDOM_INFO_OUTER]
        if self._.get(HDOM_INFO_TEXT) :
            return self.dom.restore_noise(self._[HDOM_INFO_TEXT])

        # render begin tag
        #print(self._[HDOM_INFO_BEGIN])
        ret = self.dom.nodes[self._[HDOM_INFO_BEGIN]].makeup()

        # render inner text
        if self._.get(HDOM_INFO_INNER):
            ret+=self._[HDOM_INFO_INNER]
        else:
            for n in self.nodes:
                ret+= n.outertext()


        # render end tag
        if self._.get(HDOM_INFO_END):
            if self._[HDOM_INFO_END]!=0:
                ret+= '</'+self.tag+'>'
        return ret.replace('/>>','/>')


    # get dom node's plain text
    def text(self):
        if self._.get(HDOM_INFO_INNER):
            return self._[HDOM_INFO_INNER]
        if self.nodetype==HDOM_TYPE_TEXT:
            return self.dom.restore_noise(self._[HDOM_INFO_TEXT])
        if self.nodetype==HDOM_TYPE_COMMENT or self.nodetype==HDOM_TYPE_UNKNOWN:
            return ''
        if self.tag.lower()=='script' or self.tag.lower()=='style':
            return ''
        ret = ''
        for n in self.nodes:
            ret += n.text()
        return ret


    def xmltext(self):
        ret = self.innertext()
        ret = ret.replace('<![CDATA[', '')
        ret = ret.replace(']]>', '')
        return ret


    # build node's text with tag
    def makeup(self):
        # text, comment, unknown
        if self._.get(HDOM_INFO_TEXT):
            return self.dom.restore_noise(self._[HDOM_INFO_TEXT])

        ret = '<'+self.tag
        i = -1
        for key,val in self.attr.items():
            i+=1
            # skip removed attribute
            if val==None or val==False:
                continue

            ret+= self._[HDOM_INFO_SPACE][i][0]
            #no value attr: nowrap, checked selected...
            if val==True:
                ret+=key
            else:
                quote=''
                if self._[HDOM_INFO_QUOTE][i]==HDOM_QUOTE_DOUBLE:
                    quote = '"'
                if self._[HDOM_INFO_QUOTE][i]==HDOM_QUOTE_SINGLE:
                    quote = '\''
                ret += key+self._[HDOM_INFO_SPACE][i][1]+'='+self._[HDOM_INFO_SPACE][i][2]+quote+val+quote


        ret = self.dom.restore_noise(ret)
        return ret+self._[HDOM_INFO_ENDSPACE]+'>'


    # find elements by css selector
    def find(self,selector, idx=None):
        #TEST
        #print(selector)
        selectors = self.parse_selector(selector)
        #TEST
        #print(selectors)
        count=len(selectors)
        if count==0:
            return []
        found_keys = {}

        # find each selector
        for c in range(count):
            #print(selectors[0])
            level=len(selectors[0])
            if level==0:
                return []
            if self._.get(HDOM_INFO_BEGIN)==None:
                #print('TEST:',self._.get(HDOM_INFO_BEGIN))
                return []

            head ={self._[HDOM_INFO_BEGIN]:1}

            # handle descendant selectors, no recursive!
            for l in range(level):
                ret = {}
                for k,v in head.items():
                    #??
                    if k==-1:
                        n=self.dom.root
                    else:
                        n=self.dom.nodes[k]
                    #print('in level:',selectors[c][l])
                    n.seek(selectors[c][l], ret)
                head = ret
                #print('331:',ret)

            for k,v in head.items():
                if found_keys.get(k)==None:
                    found_keys[k]=1
        found = []
        for k,v in found_keys.items():
            found.append(self.dom.nodes[k])

        #return nth-element or array
        if idx is None:
            return found
        elif idx < 0:
            idx = len(found) + idx
        try:
            return found[idx]
        except:
            return None

    # seek for given conditions
    def seek(self,selector,ret={}):
        tag, key, val, exp, no_key= selector
        #print('352:',tag, key, val, exp, no_key)
        # xpath index
        if tag!='' and key and key.isnumeric():
            count = 0
            for c in self.children:
                if tag=='*' or tag==c.tag:
                    count+=1
                    if count==key:
                        ret[c._[HDOM_INFO_BEGIN]] = 1
                        return None
            return None

        if self._.get(HDOM_INFO_END):
            end=self._[HDOM_INFO_END]
        else:
            end=0
        #print('368:',end)
        if end==0:
            parent = self.parent
            while parent._.get(HDOM_INFO_END)==None and parent!=None:
                end -= 1
                parent = parent.parent

            end += parent._[HDOM_INFO_END]
        #print('376:',self._[HDOM_INFO_BEGIN]+1,end)
        for i in range(self._[HDOM_INFO_BEGIN]+1,end):
            node = self.dom.nodes[i]
            passed = True

            if tag=='*' and not key:
                if node in self.children:
                    ret[i] = 1
                    #print('384',i)
                continue


            # compare tag
            if tag and tag!=node.tag and tag!='*':
                passed=False
            # compare key
            if passed and key:
                if no_key:
                    if node.attr.get(key)!=None:
                        passed=False

                elif node.attr.get(key)==None:
                    passed=False

            # compare value
            if passed and key and val and val!='*':
                check = self.match(exp, val, node.attr.get(key))
                # handle multiple class
                #print('405:',check)
                if not check and key.lower()=='class':
                    for k in node.attr.get(key).split(' '):
                        check = self.match(exp, val, k)
                        if check:
                            break

                if not check:
                    passed = False

            if passed:
                ret[i] = 1
            del node


    # preg_quote used to add a slash to REG  letter
    def preg_quote(self,pattern):
        regBar=r'.^$*+?{[]\|()'
        for rg in regBar:
            pattern=pattern.replace(rg,'\\'+rg)
        #print('423',pattern)
        return pattern


    def match(self,exp, pattern, value):
        if exp=='=':
            return value==pattern
        if exp=='!=':
            return value!=pattern
        if exp=='^=':
            return re.search('^'+self.preg_quote(pattern),value)
        if exp=='$=':
            return re.search(self.preg_quote(pattern)+'$',value)
        if exp=='*=':
            if pattern[0]=='/':
                return re.search(pattern,value)
            else:
                return re.search(pattern, value,re.I)
        return False

    #MYMARK
    def parse_selector(self,selector_string):
        # pattern of CSS selectors, modified from mootools
        pattern = r"(([\w\-\:\*]*)(?:\#([\w\-]+)|\.([\w\-]+))?(?:\[@?(!?[\w\-]+)(?:([!*^]?=)[\"']?(.*?)[\"']?)?\])?([\/, ]+))"
        matches=re.findall(pattern,selector_string.strip()+' ',re.I)
        selectors = []
        result = []
        #print(matches)

        for m in matches:
            #print(m)
            m0 = m[0].strip()
            if m0=='' or m0=='/' or m0=='#':
                continue
            # for borwser grnreated xpath
            if m[1]=='tbody':
                continue

            tag, key, val, exp, no_key=(m[1], None, None, '=', False)
            if m[2]!='':
                key='id'
                val=m[2]
            if m[3]!='':
                key='class'
                val=m[3]
            if m[4]!='':
                key=m[4]
            if m[5]!='':
                exp=m[5]
            if m[6]!='':
                val=m[6]

            # convert to lowercase

            if self.dom.lowercase:
                tag=tag.lower()
                if key!=None:
                    key=key.lower()

            #elements that do NOT have the specified attribute
            if key is not None:
                if key[0]=='!':
                    key=key[1:]
                    no_key=True

            result.append([tag, key, val, exp, no_key])
            if m[7].strip()==',':
                selectors.append(result)
                result=[]


        if len(result)>0:
            selectors.append(result)
        return selectors


    def __getattr__(self,name):
        if self.attr.get(name)!=None:
            return self.attr[name]
        if name=='title':
            return self.attr.get('itle')

        if name=='outertext':
            return self.outertext()
        if name=='innertext':
            return self.innertext()
        if name=='plaintext':
            return self.text()
        if name=='xmltext':
            return self.xmltext()
        #print(name)
        try:
            returnValue=object.__getattribute__(self,name)
        except:
            returnValue=None
        return returnValue



    def __setattr__(self,name, value):
        #print(name)
        if name=='outertext':
            self._[HDOM_INFO_OUTER] = value
            return value
        if name=='innertext':
            if  self._.get(HDOM_INFO_TEXT):
                self._[HDOM_INFO_TEXT] = value
                return value
            self._[HDOM_INFO_INNER] = value
            return value
        if name not in ('tag','dom','_','parent','children','nodetype','attr','__id__','nodes'):
            if self.attr.get(name)==None:
                self._[HDOM_INFO_SPACE]=[]
                self._[HDOM_INFO_QUOTE]=[]
                self._[HDOM_INFO_SPACE].append([' ', '', ''])
                self._[HDOM_INFO_QUOTE].append(HDOM_QUOTE_DOUBLE)
            else:
                self.attr[name] = value
        elif name in ('tag','dom','_','parent','children','nodetype','attr','__id__','nodes'):
            #print('self attr:'+name)
            object.__setattr__(self,name,value)


    def __delattr__(self,name):
        if self.attr.get(name)!=None:
            del self.attr[name]
        object.__delattr__(self,name)

    def __eq__(self,other):
        if other==None:
            return False
        if self.getId()==other.getId():
            return True
        return False
    def __ne__(self,other):
        return not self.__eq__(other)

    # unique nodes identify
    def getId(self):
        return self.__id__

    # camel naming conventions
    def getAllAttributes(self):
        return self.attr
    def getAttribute(self,name):
        return self.__getattr__(name)
    def setAttribute(self,name, value):
        self.__setattr__(name, value)
    def hasAttribute(self,name):
        return self.__contains__(name)
    def removeAttribute(self,name):
        self.__setattr__(name, None)
    def getElementById(self,id):
        return self.find("#"+id, 0)
    def getElementsById(self,id, idx=None):
        return self.find("#"+id, idx)
    def getElementByTagName(self,name):
        return self.find(name, 0)
    def getElementsByTagName(self,name, idx=None):
        return self.find(name, idx)
    def parentNode(self):
        return self.get_parent()
    def childNodes(self,idx=-1):
        return self.child(idx)
    def firstChild(self):
        return self.first_child()
    def lastChild(self):
        return self.last_child()
    def nextSibling(self):
        return self.next_sibling()
    def previousSibling(self):
        return self.prev_sibling()



# simple html dom parser
# -----------------------------------------------------------------------------
class simple_html_dom(object):
    ''' simple_html_dom class,this is the main class'''

    def __init__(self,str=None):
        self.root = None
        self.nodes = []
        self.callback = None
        self.lowercase = False
        self.pos=0
        self.doc=None
        self.char=None
        self.size=None
        self.cursor=None
        self.parent=None
        self.noise ={}
        self.token_blank = " \r\n\t"
        self.token_equal = " =/>"
        self.token_slash = " />\r\n\t"
        self.token_attr = " >"
        # use isset instead of in_array, performance boost about 30%...
        self.self_closing_tags ={'img':1, 'br':1, 'input':1, 'meta':1, 'link':1, 'hr':1, 'base':1, 'embed':1, 'spacer':1}
        self.block_tags = {'root':1, 'body':1, 'form':1, 'div':1, 'span':1, 'table':1}
        self.optional_closing_tags ={
            'tr':{'tr':1, 'td':1, 'th':1},
            'th':{'th':1},
            'td':{'td':1},
            'li':{'li':1},
            'dt':{'dt':1, 'dd':1},
            'dd':{'dd':1, 'dt':1},
            'dl':{'dd':1, 'dt':1},
            'p':{'p':1},
            'nobr':{'nobr':1},
        }
        if str:
            if str.startswith('http://') or os.path.isfile(str):
                self.load_file(str)
            else:
                self.load(str)
        #print('Dom',id(self),'Born')


    def __del__(self):
        self.clear()
        #TEST
        #print('Object Delelted')
        #print('Dom',id(self),'Died')
        #object.__del__(self)


    # load html from string
    def load(self,str,lowercase=True):
        # prepare
        self.prepare(str, lowercase)
        # strip out comments
        self.remove_noise(r'(<!--(.*?)-->)')
        # strip out cdata
        self.remove_noise(r'(<!\[CDATA\[(.*?)\]\]>)', True)
        # strip out <style> tags
        self.remove_noise(r'(<\s*style[^>]*[^/]>(.*?)<\s*/\s*style\s*>)')
        self.remove_noise(r'(<\s*style\s*>(.*?)<\s*/\s*style\s*>)')
        # strip out <script> tags
        self.remove_noise(r'(<\s*script[^>]*[^/]>(.*?)<\s*/\s*script\s*>)')
        self.remove_noise(r'(<\s*script\s*>(.*?)<\s*/\s*script\s*>)')
        # strip out preformatted tags
        self.remove_noise(r'(<\s*(?:code)[^>]*>(.*?)<\s*/\s*(?:code)\s*>)')
        # strip out server side scripts
        self.remove_noise(r'((<\?)(.*?)(\?>))', True)
        # strip smarty scripts
        self.remove_noise(r'((\{\w)(.*?)(\}))', True)

        # parsing
        while self.parse():
            pass
        # end
        self.root._[HDOM_INFO_END] = self.cursor

    # load html from file
    def load_file(self,filename):
        self.load(loadData(filename), True)


    # set callback def
    def set_callback(self,def_name):
        self.callback = def_name


    # remove callback def
    def remove_callback(self):
        self.callback = None


    # save dom as string
    def save(self,filepath=''):
        ret = self.root.innertext()
        if filepath!='':
            try:
                with open(filepath,'w',encoding='utf-8') as f:
                    f.write(ret)
                    f.close()
            except:
                pass
        return ret

    # find dom node by css selector
    def find(self,selector, idx=None):
        return self.root.find(selector, idx)


    # clean up memory due to php5 circular references memory leak...
    def clear(self):
        #print('Object Delelted')
        for n in self.nodes:
             n.clear()
             n = None
        self.nodes=[]
        if self.parent!=None:
            self.parent.clear()
            self.parent=None
        if self.root!=None:
            self.root.clear()
            self.root=None
        self.doc=None
        self.noise={}


    def dump(self,show_attr=True):
        self.root.dump(show_attr)


    # prepare HTML data and init everything
    def prepare(self,str, lowercase=True):
        self.clear()
        self.doc = str
        self.pos = 0
        self.cursor = 1
        self.noise = {}
        self.nodes = []
        self.lowercase = lowercase
        self.root =simple_html_dom_node(self)
        self.root.tag = 'root'
        self.root._[HDOM_INFO_BEGIN] = -1
        self.root.nodetype = HDOM_TYPE_ROOT
        self.parent = self.root
        # set the length of content
        self.size = len(str)
        if self.size>0:
            self.char = self.doc[0]


    # parse html content
    def parse(self):
        s=self.copy_until_char('<')
        if s=='':
            return self.read_tag()

        # text
        node =simple_html_dom_node(self)
        self.cursor+=1
        node._[HDOM_INFO_TEXT] = s
        self.link_nodes(node, False)
        return True


    # read tag info
    def read_tag(self):
        if self.char!='<':
            self.root._[HDOM_INFO_END] = self.cursor
            return False

        begin_tag_pos = self.pos
        self.pos+=1
        if self.pos<self.size:
            self.char=self.doc[self.pos]
        else:
            self.char=None

        # end tag
        if self.char=='/':
            self.pos+=1
            if self.pos<self.size:
                self.char=self.doc[self.pos]
            else:
                self.char=None
            self.skip(self.token_blank)
            tag = self.copy_until_char('>')

            # skip attributes in end tag
            pos=tag.find(' ')
            if pos!=-1:
                tag =tag[0:pos]

            parent_lower =self.parent.tag.lower()
            tag_lower = tag.lower()

            if parent_lower!=tag_lower:
                if self.optional_closing_tags.get(parent_lower) and self.block_tags.get(tag_lower):
                    self.parent._[HDOM_INFO_END] = 0
                    org_parent = self.parent

                    while self.parent.parent and self.parent.tag.lower()!=tag_lower:
                        self.parent = self.parent.parent

                    if self.parent.tag.lower()!=tag_lower:
                        self.parent = org_parent # restore origonal parent
                        if self.parent.parent:
                            self.parent = self.parent.parent
                        self.parent._[HDOM_INFO_END] = self.cursor
                        return self.as_text_node(tag)

                elif self.parent.parent and self.block_tags.get(tag_lower):
                    self.parent._[HDOM_INFO_END] = 0
                    org_parent = self.parent

                    while self.parent.parent and self.parent.tag.lower()!=tag_lower:
                        self.parent = self.parent.parent

                    if self.parent.tag.lower()!=tag_lower:
                        self.parent = org_parent # restore origonal parent
                        self.parent._[HDOM_INFO_END] = self.cursor
                        return self.as_text_node(tag)


                elif self.parent.parent and self.parent.parent.tag.lower()==tag_lower:
                    self.parent._[HDOM_INFO_END] = 0
                    self.parent = self.parent.parent
                else:
                    return self.as_text_node(tag)


            self.parent._[HDOM_INFO_END] = self.cursor
            if self.parent.parent:
                self.parent = self.parent.parent

            self.pos+=1
            if self.pos<self.size:
                self.char=self.doc[self.pos]
            else:
                self.char=None
            return True


        node =simple_html_dom_node(self)
        node._[HDOM_INFO_BEGIN] = self.cursor
        self.cursor+=1
        tag = self.copy_until(self.token_slash)

        # doctype, cdata & comments...

        if tag.__len__()>0:
            if tag[0]=='!':
                node._[HDOM_INFO_TEXT] = '<' +tag+self.copy_until_char('>')

                if tag.__len__()>2:
                    if tag[1]=='-' and tag[2]=='-':
                        node.nodetype = HDOM_TYPE_COMMENT
                        node.tag = 'comment'
                    else:
                        node.nodetype = HDOM_TYPE_UNKNOWN
                        node.tag = 'unknown'
                else:
                    node.nodetype = HDOM_TYPE_UNKNOWN
                    node.tag = 'unknown'


                if self.char=='>':
                    node._[HDOM_INFO_TEXT]+='>'
                self.link_nodes(node, True)
                self.pos+=1
                if self.pos<self.size:
                    self.char=self.doc[self.pos]
                else:
                    self.char=None
                return True


        # text
        pos=tag.find('<')
        if pos!=-1:
            tag = '<'+tag[0:-1]
            node._[HDOM_INFO_TEXT] = tag
            self.link_nodes(node, False)
            self.pos-=1
            self.char = self.doc[self.pos] # prev
            return True


        if re.search(r"^[\w\-:]+$", tag)==None:
            node._[HDOM_INFO_TEXT] = '<'+tag+self.copy_until('<>')
            if self.char=='<':
                self.link_nodes(node, False)
                return True


            if self.char=='>':
                node._[HDOM_INFO_TEXT]+='>'
            self.link_nodes(node, False)
            self.pos+=1
            if self.pos<self.size:
                self.char=self.doc[self.pos]
            else:
                self.char=None
            return True


        # begin tag
        node.nodetype = HDOM_TYPE_ELEMENT
        tag_lower = tag.lower()
        if self.lowercase:
            node.tag =tag_lower
        else:
            node.tag =tag


        # handle optional closing tags
        if self.optional_closing_tags.get(tag_lower):
            while self.optional_closing_tags.get(tag_lower).get(self.parent.tag.lower()):
                self.parent._[HDOM_INFO_END] = 0
                self.parent = self.parent.parent

            node.parent = self.parent


        guard = 0 # prevent infinity loop
        space =[self.copy_skip(self.token_blank), '', '']

        # attributes
        while True:
            if self.char!=None and space[0]=='':
                break
            name = self.copy_until(self.token_equal)
            if guard==self.pos:
                self.pos+=1
                if self.pos<self.size:
                    self.char =self.doc[self.pos]
                else:
                    self.char =None
                continue

            guard = self.pos

            # handle endless '<'
            if self.pos>=self.size-1 and self.char!='>':
                node.nodetype = HDOM_TYPE_TEXT
                node._[HDOM_INFO_END] = 0
                node._[HDOM_INFO_TEXT] = '<'+tag +space[0]+name
                node.tag = 'text'
                self.link_nodes(node, False)
                return True

            # handle mismatch '<'
            if self.doc[self.pos-1]=='<':
                node.nodetype = HDOM_TYPE_TEXT
                node.tag = 'text'
                node.attr ={}
                node._[HDOM_INFO_END] = 0
                node._[HDOM_INFO_TEXT] = self.doc[begin_tag_pos:self.pos-1]
                self.pos -= 2
                self.pos+=1
                if self.pos<self.size:
                    self.char =self.doc[self.pos]
                else:
                    self.char =None
                self.link_nodes(node, False)
                return True


            if name!='/' and name!='':
                space[1] = self.copy_skip(self.token_blank)
                name = self.restore_noise(name)
                if self.lowercase:
                    name = name.lower()
                if self.char=='=':
                    self.pos+=1
                    if self.pos<self.size:
                        self.char =self.doc[self.pos]
                    else:
                        self.char =None
                    self.parse_attr(node, name, space)

                else:
                    #no value attr: nowrap, checked selected...
                    if node._.get(HDOM_INFO_QUOTE)==None:
                        node._[HDOM_INFO_QUOTE]=[]
                    node._[HDOM_INFO_QUOTE].append(HDOM_QUOTE_NO)
                    node.attr[name] = True
                    if self.char!='>':
                        self.pos-=1
                        self.char = self.doc[self.pos] # prev

                if node._.get(HDOM_INFO_SPACE)==None:
                    node._[HDOM_INFO_SPACE]=[]

                node._[HDOM_INFO_SPACE].append(space)
                space =[self.copy_skip(self.token_blank), '', '']

            else:
                break
            if self.char=='>' or self.char=='/':
                break


        self.link_nodes(node, True)
        node._[HDOM_INFO_ENDSPACE] = space[0]

        # check self closing
        if self.copy_until_char_escape('>')=='/':
            node._[HDOM_INFO_ENDSPACE]+= '/'
            node._[HDOM_INFO_END] = 0
        else:
            # reset parent
            if self.self_closing_tags.get(node.tag.lower())==None:
                self.parent = node
            self.pos+=1
            if self.pos<self.size:
                self.char =self.doc[self.pos]
            else:
                self.char =None
        return True


    # parse attributes
    def parse_attr(self,node, name, space=[]):
        space[2] = self.copy_skip(self.token_blank)
        if self.char=='"':
            if node._.get(HDOM_INFO_QUOTE)==None:
                node._[HDOM_INFO_QUOTE]=[]
            node._[HDOM_INFO_QUOTE].append(HDOM_QUOTE_DOUBLE)
            self.pos+=1
            if self.pos<self.size:
                self.char =self.doc[self.pos]
            else:
                self.char =None
            node.attr[name] = self.restore_noise(self.copy_until_char_escape('"'))
            self.pos+=1
            if self.pos<self.size:
                self.char =self.doc[self.pos]
            else:
                self.char =None
        elif self.char=="'":
            if node._.get(HDOM_INFO_QUOTE)==None:
                node._[HDOM_INFO_QUOTE]=[]
            node._[HDOM_INFO_QUOTE].append(HDOM_QUOTE_SINGLE)
            self.pos+=1
            if self.pos<self.size:
                self.char =self.doc[self.pos]
            else:
                self.char =None
            node.attr[name] = self.restore_noise(self.copy_until_char_escape("'"))
            self.pos+=1
            if self.pos<self.size:
                self.char =self.doc[self.pos]
            else:
                self.char =None
        else:
            if node._.get(HDOM_INFO_QUOTE)==None:
                node._[HDOM_INFO_QUOTE]=[]
            node._[HDOM_INFO_QUOTE].append(HDOM_QUOTE_NO)
            node.attr[name] = self.restore_noise(self.copy_until(self.token_attr))



    # link node's parent
    def link_nodes(self,node, is_child):
        node.parent = self.parent
        self.parent.nodes.append(node)
        if is_child:
            self.parent.children.append(node)


    # as a text node
    def as_text_node(self,tag):
        node = simple_html_dom_node(self)
        self.cursor+=1
        node._[HDOM_INFO_TEXT] = '</' +tag+'>'
        self.link_nodes(node, False)
        self.pos+=1
        if self.pos<self.size:
            self.char =self.doc[self.pos]
        else:
            self.char =None
        return True

    def skip(self,chars):
        self.pos += strspn(self.doc, chars, self.pos)
        if self.pos<self.size:
            self.char =self.doc[self.pos]
        else:
            self.char =None


    def copy_skip(self,chars):
        pos = self.pos
        len = strspn(self.doc, chars, pos)
        self.pos += len
        if self.pos<self.size:
            self.char =self.doc[self.pos]
        else:
            self.char =None
        if len==0:
            return ''
        return self.doc[pos:pos+len]


    def copy_until(self,chars):
        pos = self.pos
        len = strcspn(self.doc, chars, pos)
        self.pos += len
        if self.pos<self.size:
            self.char =self.doc[self.pos]
        else:
            self.char =None
        return self.doc[pos:pos+len]


    def copy_until_char(self,char):
        if self.char==None:
            return ''
        pos =self.doc.find(char,self.pos)
        if pos==-1:
            ret =self.doc[self.pos:self.size]
            self.char = None
            self.pos = self.size
            return ret


        if pos==self.pos:
            return ''
        pos_old = self.pos
        self.char = self.doc[pos]
        self.pos = pos
        return self.doc[pos_old:pos]


    def copy_until_char_escape(self,char):
        if self.char==None:
            return ''

        start = self.pos
        while True:
            pos =self.doc.find(char,start)
            if pos==-1:
                ret = self.doc[self.pos:self.size]
                self.char = None
                self.pos = self.size
                return ret


            if pos==self.pos:
                return ''

            if self.doc[pos-1]=="\\":
                start = pos+1
                continue

            pos_old = self.pos
            self.char = self.doc[pos]
            self.pos = pos
            return self.doc[pos_old:pos]

    # remove noise from html content
    def remove_noise(self,pattern, remove_tag=False):
        matches =re.findall(pattern, self.doc,re.I|re.S)
        count=matches.__len__()
        i=count-1
        #print('Line:1190',count,matches)
        while i>-1:
            key = '___noise___'+'%3d'%(len(self.noise)+100)
            if remove_tag:
                idx=1
                #idx=0
            else:
                idx=0
            try:
                self.noise[key] = matches[i][idx]
            except:
                 #print(matches)
                 pass
            #print(self.doc)
            try:
                self.doc =self.doc.replace(matches[i][idx],key)
            except:
                #print(matches)
                pass
            i-=1

        # reset the length of content
        self.size = len(self.doc)
        if self.size>0:
            self.char = self.doc[0]


    # restore noise to html content
    def restore_noise(self,text):
        pos=text.find('___noise___')
        while pos!=-1:
            key = '___noise___'+text[pos+11]+text[pos+12]+text[pos+13]
            if self.noise.get(key):
                text =text[0:pos]+self.noise.get(key)+text[pos+14:]
            pos=text.find('___noise___')
        return text


    def __str__(self):
        return self.root.innertext()


    def __getattr__(self,name):
        if name=='outertext':
            return self.root.outertext()
        if name=='innertext':
            return self.root.innertext()
        if name=='plaintext':
            return self.root.text()
        return object.__getattribute__(self,name)

    # camel naming conventions
    def childNodes(self,idx=-1):
        return self.root.childNodes(idx)
    def firstChild(self):
        return self.root.first_child()
    def lastChild(self):
        return self.root.last_child()
    def getElementById(self,id):
        return self.find("#"+id, 0)
    def getElementsById(self,id, idx=None):
        return self.find("#"+id, idx)
    def getElementByTagName(self,name):
        return self.find(name, 0)
    def getElementsByTagName(self,name, idx=-1):
        return self.find(name, idx)
    def loadFile(self,filename):
        self.load(loadData(filename), True)
#==========================================================end================================================